﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using WebSiteBase;
using DME.Web.Net;
using System.Windows.Forms;
using System.Threading;
using System.Xml;
using System.Collections;

namespace IYAA_BBS_SPIDER
{
    public class Html
    {

        private Html() { }
        private static Html _Html = null;
        public static Html GetNewInstnce()
        {
            if (_Html == null)
            {
                _Html = new Html();
            }
            return _Html;
        }



        //获得html源码
        public string GetResponseString(string url)
        {
            DMEWeb_HttpClient http = null;
            string _StrResponse = string.Empty;
            try
            {
                http = new DMEWeb_HttpClient();
                http.Url = url;
                http.Verb = DMEWeb_HttpVerb.GET;
                _StrResponse = http.GetString();
                http = null;

                //if (_StrResponse == "")
                //{
                //    StreamReader _ResponseStream = null;
                //    HttpWebRequest _WebRequest = (HttpWebRequest)WebRequest.Create(url);
                //    _WebRequest.UserAgent = "MOZILLA/4.0 (COMPATIBLE; MSIE 7.0; WINDOWS NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)";
                //    _WebRequest.Method = "GET";
                //    _WebRequest.Timeout = 30000;
                //    WebResponse _WebResponse = _WebRequest.GetResponse();
                //    _ResponseStream = new StreamReader(_WebResponse.GetResponseStream(), Encoding.UTF8);
                //    Encoding enc = GetEncoding(_ResponseStream);
                //    _StrResponse = _ResponseStream.ReadToEnd();
                //    _WebResponse.Close();
                //    _ResponseStream.Close();
                //    _ResponseStream.Dispose();
                    

                //    string _StrResponse2 = string.Empty;
                //    StreamReader _ResponseStream2 = null;
                //    HttpWebRequest _WebRequest2 = (HttpWebRequest)WebRequest.Create(url);
                //    _WebRequest2.UserAgent = "MOZILLA/4.0 (COMPATIBLE; MSIE 7.0; WINDOWS NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)";
                //    _WebRequest2.Method = "GET";
                //    _WebRequest2.Timeout = 30000;
                //    WebResponse _WebResponse2 = _WebRequest2.GetResponse();
                //    _ResponseStream2 = new StreamReader(_WebResponse2.GetResponseStream(), enc);
                //    _StrResponse2 = _ResponseStream2.ReadToEnd();
                //    _WebResponse2.Close();
                //    _ResponseStream2.Close();
                //    _ResponseStream2.Dispose();

                //    _StrResponse = _StrResponse2;


                //}
            }
            catch (Exception ex) { StringHelper.WriteLogger(ex); }
            return _StrResponse == "" ? "" : _StrResponse.Replace("\n", "").Replace("\r", "").Replace("\t", "").Replace("  ", "").Replace("&nbsp;", "");
        }

        // 根据网页的HTML内容提取网页的Encoding
        public Encoding GetEncoding(StreamReader sr)
        {
            string html = sr.ReadToEnd();

            string pattern = @"(?is)<meta[^>]*?charset=(['""\s]?)(?<charset>[^'""\s]*)\1[^>]*?>";
            string charset = Regex.Match(html, pattern).Groups["charset"].Value;
            try { return Encoding.GetEncoding(charset); }
            catch (ArgumentException) { return null; }
        }




        //取分类
        public string getCate(string html, string rege, int num)
        {
            Regex reg = null;
            string[] cateArr = null;
            string cate = null;
            cate = "";
            reg = new Regex(rege);
            MatchCollection result = reg.Matches(html);
            foreach (Match m in result)
            {
                Regex objRegExp = new Regex("<(?!/a|/A)[^>]*>");
                string tem = objRegExp.Replace(m.Value, "");
                tem = tem.Replace("\n", "").Replace("\r", "").Replace("\t", "").Replace("&nbsp;", "")
                    .Replace(">>", "").Replace("&gt;", "").Replace(" ", "").Replace("</a>", "|")
                    .Replace("&raquo;", "").Replace("『天河』→", "").Replace("『", "").Replace("』", "").Replace("->", "");
                tem = tem.TrimEnd('|').TrimStart('|');
                cateArr = tem.Split('|');
                cate = cateArr[num].Replace("-", "");

            }
            return cate;
        }


        //取得分页数量
        public int morePage(string rege, string html)
        {
            int max = 1;
            Regex reg = null;
            reg = new Regex(rege);
            MatchCollection result = reg.Matches(html);
            foreach (Match m in result)
            {
                Regex objRegExp = new Regex("<(.|\n)+?>");
                string tem = objRegExp.Replace(m.Value, "");
                tem = tem.Replace(" ", "").Replace(">", "").Replace("下一页", "").Replace(".", "").Replace("共", "").Replace("页", "");
                //tem = tem.Substring(tem.IndexOf("["), tem.IndexOf("]") - tem.IndexOf("[")).Replace("[", "");
                max = Convert.ToInt32(tem);
            }
            return max;
        }


        //取标题等..
        public string getPro(string html, string rege)
        {
            string Title = "";
            Regex reg = new Regex(rege);
            MatchCollection result = reg.Matches(html);
            if (result != null && result.Count != 0)
            {
                Regex objRegExp = new Regex("<(.|\n)+?>");
                Title = objRegExp.Replace(result[0].Value, "").Replace("&nbsp;", " ").Replace("&lt;", "<").Replace("&gt;", ">");
                Title = Title.Replace("\n", "").Replace("\r", "").Replace("\t", "").Trim();
            }
            return Title;
        }

        /// <summary>
        /// 过滤html
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
        public string HtmlFilter(string html)
        {
            Regex objRegExp = new Regex("<(?!img|IMG)[^>]*>");//<(.|\n)+?>
            html = objRegExp.Replace(html, "").Replace("&nbsp;", " ").Replace("&lt;", "<").Replace("&gt;", ">");
            html = html.Replace("\n", "").Replace("\r", "").Replace("\t", "").Trim();
            return html;
        }


        public string GetWebContent(string Url)
        {
            string strResult = "";
            try
            {
                Encoding encoding;
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
                //声明一个HttpWebRequest请求
                request.Timeout = 30000;
                //设置连接超时时间
                request.Headers.Set("Pragma", "no-cache");
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                Stream streamReceive = response.GetResponseStream();
                encoding = Encoding.UTF8;
                StreamReader streamReader = new StreamReader(streamReceive, encoding);
                strResult = streamReader.ReadToEnd();
            }

            catch (Exception ex) { StringHelper.WriteLogger(ex); }
            return strResult;
        }



        //取得分页数量
        public string GetNextPageUrl(string rege, string html, string url)
        {
            string n_url = string.Empty;
            string n_Doma = string.Empty;

            Regex reg = new Regex(rege);
            MatchCollection result = reg.Matches(html);
            if (result != null && result.Count != 0)
            {
                string purl = result[0].Groups["href"].ToString().Replace("../", "").Replace("./", "").Replace("~/", "").TrimStart('/');

                if (purl.StartsWith("http://"))
                {
                    n_url = purl;
                }
                else if (purl.StartsWith("?"))
                {
                    if (url.IndexOf("?") > -1)
                    {
                        n_url = url.Substring(0, url.IndexOf("?"));
                        n_url = n_url + purl;
                    }

                }
                else
                {
                    string domain = url.Substring(0, url.Replace("http://", "").IndexOf("/") + 8);
                    string domain2 = url.Substring(0, url.LastIndexOf("/") + 1);


                    if (domain == domain2)
                    {
                        n_url = domain + purl;
                    }
                    else
                    {
                        string folder = domain2.Substring(domain.Length, domain2.Length - domain.Length);
                        string folder2 = n_url.IndexOf("/") > -1 ? n_url.Substring(0, n_url.IndexOf("/") + 1) : "";
                        n_url = folder2 == "" ? domain2 + n_url : domain2.Replace(folder, "") + n_url;

                    }

                    //n_url = (url.Substring(0, url.Replace("http://", "").IndexOf("/") + 8) + n_url).Replace("../", "/").Replace("./", "/").Replace("~/", "/").Replace("//", "/").Replace("http:", "http://");

                    //n_Doma = url.Substring(0, url.IndexOf(".com") + 4);
                    //n_url = n_Doma + m.Groups["href"].ToString();
                }



            }
            return n_url.Replace("\n", "").Replace("\r", "").Replace("\t", "").Replace("  ", "").Replace("&nbsp;", "");
        }



        public string GetNormal(string html, string startKW, string endKW)
        {
            string NormalStr = string.Empty;
            Regex reg = new Regex("(?is)" + startKW + @"[^>]*?[^>]*?>(?<text>[^'""\s]*).*" + endKW);
            MatchCollection result = reg.Matches(html);
            if (result != null && result.Count != 0)
            {
                NormalStr = result[0].Groups["text"].ToString();
            }
            return NormalStr;

        }

        //取标题等..
        public string getNormalText(string html, string startKW, string endKW)
        {
            string Title = "";
            Regex reg = new Regex("(?<=" + startKW + @")[\s\S]*?(?=" + endKW + ")");
            MatchCollection result = reg.Matches(html);
            if (result != null && result.Count != 0)
            {
                Regex objRegExp = new Regex("<(.|\n)+?>");
                Title = objRegExp.Replace(result[0].Value.Replace("</p>", "/"), "").Replace("&nbsp;", " ").Replace("&lt;", "<").Replace("&gt;", ">").Replace("<", "").Replace(">", "").Trim('/');
                Title = Title.Trim().Replace("&yen;", "").Replace("¥", "").Replace("￥", "").Replace("\"", "");
            }
            return Title;
        }


        public string GetNormalUrl(string html, string startKW, string endKW)
        {
            string NormalStr = string.Empty;
            Regex reg = new Regex("(?is)" + startKW + @"[^>]*?href=(['""\s]?)(?<href>[^'""\s]*)\1[^>]*?" + endKW);
            MatchCollection result = reg.Matches(html);
            if (result != null && result.Count != 0)
            {
                NormalStr = result[0].Groups["href"].ToString();
            }
            return NormalStr;
        }


        public string GetImg(int sid, string html, string startKW, string endKW, string bname, string lname, string url)
        {
            string img = string.Empty;
            string path = string.Empty;
            string folder = DateTime.Now.ToShortDateString().Replace("/", "-") + "_" + sid + "\\" + bname + "\\" + lname;
            if (!Directory.Exists("image\\" + folder))
            {
                Directory.CreateDirectory("image\\" + folder);
            }
            Regex reg = new Regex("(?is)" + startKW + @"[^>]*?src=(['""\s]?)(?<src>[^'""\s]*)\1[^>]*?" + endKW);
            MatchCollection result = reg.Matches(html);
            if (result != null && result.Count != 0)
            {
                string s = result[0].Groups["src"].ToString();
                s = s.IndexOf("?") > -1 ? s.Substring(0, s.IndexOf("?")) : s;
                //s = s.IndexOf("http://") > -1 ? s : url.Substring(0, url.Replace("http://", "").IndexOf("/") + 8) + s;


                s = s.IndexOf("http://") > -1 ? s : (url.Substring(0, url.Replace("http://", "").IndexOf("/") + 8) + s).Replace("../", "/").Replace("./", "/").Replace("~/", "/").Replace("//", "/").Replace("http:", "http://");


                string filename = s.Substring(s.LastIndexOf("/"), s.Length - s.LastIndexOf("/")).Replace("/", "");
                path = s != "" ? "image\\" + folder + "\\" + filename : "";
                if (path != "" && path != null)
                {
                    try
                    {
                        WebClient wc = new WebClient();
                        wc.DownloadFile(s, path);
                    }
                    catch (Exception ex) { StringHelper.WriteLogger(ex); }
                }
            }
            return path;
        }


        public string GetAjaxPageHtml(string url)
        {
            string html = string.Empty;
            WebBrowser wb = null;
            wb = new WebBrowser();

            wb.ScriptErrorsSuppressed = true;

            wb.Navigate(url);

            try
            {
                //while (wb.StatusText != "完毕" & wb.StatusText != "完成")
                while (wb.ReadyState != WebBrowserReadyState.Complete)
                {
                    Application.DoEvents();
                    Thread.Sleep(20);
                }
                html = wb.Document.Body.InnerHtml;

                for (int i = 0; i <= 3 && html.IndexOf("不能链接到您请求的网页") > -1; i++)
                {
                    wb.Navigate(url);
                    while (html.IndexOf("不能链接到您请求的网页") > -1)
                    {
                        Application.DoEvents();
                        html = wb.Document.Body.InnerHtml;
                        Thread.Sleep(20);
                    }
                    html = wb.Document.Body.InnerHtml;
                }
                wb.Navigate("about:blank");
                //while (wb.StatusText != "完毕" & wb.StatusText != "完成")
                while (wb.ReadyState != WebBrowserReadyState.Complete)
                {
                    Application.DoEvents();
                    Thread.Sleep(20);
                }

            }
            catch
            {
                wb.Navigate("about:blank");
                //while (wb.StatusText != "完毕" & wb.StatusText != "完成")
                while (wb.ReadyState != WebBrowserReadyState.Complete)
                {
                    Application.DoEvents();
                    Thread.Sleep(20);
                }
            }
            wb.Dispose();
            wb = null;

            return html == "" ? "" : html.Replace("\n", "").Replace("\r", "").Replace("\t", "").Replace("  ", "").Replace("&nbsp;", "");
        }



        public string GetTrueUrl(string url)
        {
            string trueurl = string.Empty;
            WebBrowser wb = null;
            wb = new WebBrowser();

            wb.ScriptErrorsSuppressed = true;

            wb.Navigate(url);

            try
            {
                while (wb.StatusText != "完毕" & wb.StatusText != "完成")
                {
                    Application.DoEvents();
                    Thread.Sleep(20);
                }
                wb = new WebBrowser();
                wb.Navigate(url);
                while (wb.StatusText != "完毕" & wb.StatusText != "完成")
                {
                    Application.DoEvents();
                    Thread.Sleep(20);
                }
                trueurl = wb.Url.ToString();

                wb.Navigate("about:blank");
                while (wb.StatusText != "完毕" & wb.StatusText != "完成")
                {
                    Application.DoEvents();
                    Thread.Sleep(20);
                }
            }
            catch
            {
                wb.Navigate("about:blank");
                while (wb.StatusText != "完毕" & wb.StatusText != "完成")
                {
                    Application.DoEvents();
                    Thread.Sleep(20);
                }
            }
            wb.Dispose();
            wb = null;

            return trueurl;
        }


        /// <summary>   
        /// 通过get方式请求页面，传递一个实例化的cookieContainer   
        /// </summary>   
        /// <param name="postUrl"></param>   
        /// <param name="cookie"></param>   
        /// <returns></returns>   
        public ArrayList GetHtmlData(string postUrl, CookieContainer cookie)
        {
            HttpWebRequest request;
            HttpWebResponse response;
            ArrayList list = new ArrayList();
            request = WebRequest.Create(postUrl) as HttpWebRequest;
            request.Method = "GET";
            request.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1 QQDownload/1.7";
            request.CookieContainer = cookie;
            request.KeepAlive = true;

            request.CookieContainer = cookie;
            try
            {
                //获取服务器返回的资源   
                using (response = (HttpWebResponse)request.GetResponse())
                {
                    using (StreamReader reader = new StreamReader(response.GetResponseStream(), Encoding.Default))
                    {
                        cookie.Add(response.Cookies);
                        //保存Cookies   
                        list.Add(cookie);
                        list.Add(reader.ReadToEnd());
                        //list.Add(Guid.NewGuid().ToString());//图片名   
                        if (response.Cookies.Count != 0)
                        {
                            list.Add(response.Cookies[0]);
                        }
                    }
                }
            }
            catch (WebException ex)
            {
                list.Clear();
                list.Add("发生异常/n/r");
                WebResponse wr = ex.Response;
                if (wr != null)
                {
                    using (Stream st = wr.GetResponseStream())
                    {
                        using (StreamReader sr = new StreamReader(st, System.Text.Encoding.Default))
                        {
                            list.Add(sr.ReadToEnd());
                        }
                    }
                }

            }
            catch (Exception ex)
            {
                list.Clear();
                list.Add("5");
                list.Add("发生异常：" + ex.Message);
            }
            return list;
        }



        /// <summary>   
        /// 发送相关数据至页面   
        /// 进行登录操作   
        /// 并保存cookie   
        /// </summary>   
        /// <param name="postData"></param>   
        /// <param name="postUrl"></param>   
        /// <param name="cookie"></param>   
        /// <returns></returns>   
        public ArrayList PostData(string postData, string postUrl, CookieContainer cookie)
        {
            ArrayList list = new ArrayList();
            HttpWebRequest request;
            HttpWebResponse response;
            ASCIIEncoding encoding = new ASCIIEncoding();
            request = WebRequest.Create(postUrl) as HttpWebRequest;
            byte[] b = Encoding.UTF8.GetBytes(postData);
            request.UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1 QQDownload/1.7";
            //request.Referer = "http://jlzx.k12.com.cn/platform/app/login.php?url=http://jlzx.k12.com.cn/derup/discuz/index.php?forward=http%3A%2F%2Fwww.k12.com.cn%2F";
            request.Method = "POST";
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";

            request.ContentType = "application/x-www-form-urlencoded";
            request.CookieContainer = cookie;
            request.ContentLength = b.Length;
            using (Stream stream = request.GetRequestStream())
            {
                stream.Write(b, 0, b.Length);
            }

            try
            {
                //获取服务器返回的资源   
                using (response = request.GetResponse() as HttpWebResponse)
                {
                    using (StreamReader reader = new StreamReader(response.GetResponseStream(), Encoding.Default))
                    {
                        if (response.Cookies.Count > 0)
                            cookie.Add(response.Cookies);
                        list.Add(cookie);
                        list.Add(request.CookieContainer.GetCookieHeader(request.RequestUri)); 
                        list.Add(reader.ReadToEnd());
                        
                    }
                }
            } 
            catch (WebException wex)
            {
                list.Add(wex.Message);
                //WebResponse wr = wex.Response;
                //using (Stream st = wr.GetResponseStream())
                //{
                //    using (StreamReader sr = new StreamReader(st, System.Text.Encoding.Default))
                //    {
                //        list.Add(sr.ReadToEnd());
                //    }
                //}
            }
            catch (Exception ex)
            {
                list.Add("发生异常/n/r" + ex.Message);
            }
            return list;



        }





        /// <summary>
        /// html转ubb
        /// </summary>
        /// <param name="_Html"></param>
        /// <returns></returns>
        public string DoHtmlToUBB(string _Html)
        {
            _Html = Regex.Replace(_Html, "<br[^>]*>", "\n");
            _Html = Regex.Replace(_Html, @"<p[^>\/]*\/>", "\n");
            _Html = Regex.Replace(_Html, "\\son[\\w]{3,16}\\s?=\\s*([\'\"]).+?\\1", "");
            _Html = Regex.Replace(_Html, "<hr[^>]*>", "[hr]");

            _Html = Regex.Replace(_Html, "<(\\/)?blockquote([^>]*)>", "[$1blockquote]");
            _Html = Regex.Replace(_Html, "<img[^>]*smile=\"(\\d+)\"[^>]*>", "'[s:$1]");
            _Html = Regex.Replace(_Html, "<img\\s+[^>]*\\s*src\\s*=\\s*(['\"\"])(?<url>[^'\"\"]+)\\1[^>]*>", "[img]$2[/img]");
            _Html = Regex.Replace(_Html, "<embed [^>]*src=[\'\"\\s]*([^\\s\'\"]+)[^>]*>", "[media=swf,500,375]$1[/media]");
            _Html = Regex.Replace(_Html, "<a[^>]*href=[\'\"\\s]*([^\\s\'\"]*)[^>]*>(.+?)<\\/a>", "[url=$1]$2[/url]");
            _Html = Regex.Replace(_Html, "<[^>]*?>", "");
            _Html = Regex.Replace(_Html, "&amp;", "&");
            _Html = Regex.Replace(_Html, "&nbsp;", " ");
            _Html = Regex.Replace(_Html, "&lt;", "<");
            _Html = Regex.Replace(_Html, "&gt;", ">");

            return _Html;
        }




        /// <summary>
        /// ubb转html
        /// </summary>
        /// <param name="content"></param>
        /// <returns></returns>
        public string UBBToHtml(string content)
        {
            content = Regex.Replace(content, @"\r\n", "<br/>");
            content = Regex.Replace(content, " ", "&nbsp;");
            content = Regex.Replace(content, @"\[b\](.+?)\[/b\]", "<b>$1</b>");
            content = Regex.Replace(content, @"\[i\](.+?)\[/i\]", "<i>$1</i>");
            content = Regex.Replace(content, @"\[u\](.+?)\[/u\]", "<u>$1</u>");
            content = Regex.Replace(content, @"\[p\](.+?)\[/p\]", "<p class='load'>$1</p>");
            content = Regex.Replace(content, @"\[align=left\](.+?)\[/align\]", "<align='left'>$1</align>");
            content = Regex.Replace(content, @"\[align=center\](.+?)\[/align\]", "<align='center'>$1</align>");
            content = Regex.Replace(content, @"\[align=right\](.+?)\[/align\]", "<align='right'>$1</align>");
            content = Regex.Replace(content, @"\[url=(?<url>.+?)]\[/url]", "<a href='${url}' target=_blank>${url}</a>");
            content = Regex.Replace(content, @"\[url=(?<url>.+?)](?<name>.+?)\[/url]", "<a href='${url}' target=_blank>${name}</a>");
            content = Regex.Replace(content, @"\[quote](?<text>.+?)\[/quote]", "<div class=quote>${text}</div>");
            content = Regex.Replace(content, @"\[img](?<img>.+?)\[/img]", "<img src='${img}' alt=''/>");
            return content;
        }







    }
}
